from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from keras.models import load_model
from joblib import dump, load
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
mask = y_true != 0
return np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask]))
def fit_and_print(args, model, isNotNeural):
# Train model
X_train = args[0]
y_train = args[1]
X_test = args[2]
y_test = args[3]
if isNotNeural:
model.fit(X_train, y_train)
# Calculate predictions, MAPE, r2
y_pred = model.predict(X_test)
# Evaluate model
mape = mean_absolute_percentage_error(y_test, y_pred)
r2 = r2_score(y_true=y_test, y_pred=y_pred)
loss = mean_squared_error(y_true=y_test, y_pred=y_pred)
return loss, mape, y_pred, r2
else:
model.fit(X_train, y_train, batch_size=32, epochs=50, verbose=0)
# Evaluate model
loss = round(model.evaluate(X_test, y_test, verbose=0), 2)
# Calculate predictions and MAPE
y_pred = model.predict(X_test, verbose=0)
mape = round(mean_absolute_percentage_error(y_test, y_pred), 2)
r2 = r2_score(y_true=y_test, y_pred=y_pred)
return loss, mape, y_pred, r2
# df = pd.DataFrame(data={'prediction': y_pred.ravel(), 'actual': y_test.ravel()})
# print(df.head())
def make_double_plot(plotting_df, text, label_color_map=label_color_map):
# plot results
fig, axs = plt.subplots(1, 2, figsize=(16, 6))
for role, color in label_color_map.items():
role_data = plotting_df[plotting_df['Role'] == role]
axs[0].scatter(x=role_data['Actual'], y=role_data['Predicted'], c=color, label=role, alpha=0.5)
residuals = role_data['Predicted'] - role_data['Actual']
axs[1].hist(residuals, bins=20, color=color, alpha=0.5, label=role)
# axs[1] = sns.displot(residuals, kde=True)
axs[0].plot([0, 1], [0, 1], transform=axs[0].transAxes, ls='--', c='gray', linewidth=3)
axs[0].legend()
axs[0].set_xlim([0, 1])
axs[0].set_ylim([0, 1])
axs[0].set_xlabel('Actual')
axs[0].set_ylabel('Predicted')
titlestr = "Actual vs Predicted by Role for " + text
axs[0].set_title(titlestr)
axs[1].legend()
axs[1].set_xlim([-1, 1])
axs[1].set_xlabel('Residuals')
axs[1].set_ylabel('Frequency')
axs[1].set_title('Residual Plot')
plt.show()
def models_evaluations(args, model, text, isNotNeural=0, model_files=models_directory):
values_list_by_roles, VIF_selected_features, labels_list_by_roles, positions = args[0], args[1], args[2], args[3]
plotting_df = pd.DataFrame(columns=['Role', 'Actual', 'Predicted'])
results = []
role_models = {}
os.chdir(model_files)
print("Metrics for", text)
for i in range(0, 5):
role = positions[i]
X_train, X_test, y_train, y_test = train_test_split(values_list_by_roles[i][VIF_selected_features[i]],
labels_list_by_roles[i]['MVP'],
test_size=0.2, random_state=42)
# mini test for predicting when we remove entries with under 0.2 POG points
# yes, we understand that giving high predictions for lower pog players could
# indicate that they are underrated
X_train = X_train[y_train > 0.2]
y_train = y_train[y_train > 0.2]
X_test = X_test[y_test > 0.2]
y_test = y_test[y_test > 0.2]
# after running experiment, results obtained show that
# INSANE performance gain is obtained
# seriously considering whether to sacrifice < 0.2
# to gain huge improvements
# goes from 100~90% in neural to 25-40%
# goes from 60-70% in non-neural to 30-40%
# second mini test, above but reverse
# X_train = X_train[y_train <= 0.2]
# y_train = y_train[y_train <= 0.2]
# X_test = X_test[y_test <= 0.2]
# y_test = y_test[y_test <= 0.2]
# results show that kinda not fine cause too low results
# yeah mape is around 0.5, much better than 1.0 or 0.7
# but not really valid imo, sample size waaay too small
args_1 = [X_train, y_train, X_test, y_test]
loss, mape, y_pred, r2 = fit_and_print(args_1, model, isNotNeural)
results.append([role, loss, mape, r2])
if isNotNeural:
saved_model_name = text.replace(" ","_") + "_" +role+".joblib"
dump(model, saved_model_name)
else:
saved_model_name = text.replace(" ", "_") + "_" + role +".keras"
model.save(saved_model_name)
role_models[role] = saved_model_name
actual = y_test.values
predicted = y_pred.ravel()
plotting_df = plotting_df.append(pd.DataFrame({'Role': role,
'Actual': actual,
'Predicted': predicted}), ignore_index=True)
df = pd.DataFrame(results, columns=['Role', 'Loss', 'MAPE', 'R2 Score'])
print(df)
make_double_plot(plotting_df=plotting_df, text=text)
return role_models
